{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 用户和活动关联关系处理"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "#保存数据\n",
    "import pickle # python 3.x中cPicle模块被移除了\n",
    "\n",
    "import itertools\n",
    "\n",
    "#处理事件字符串\n",
    "import datetime\n",
    "\n",
    "import numpy as np\n",
    "import scipy.io as sio\n",
    "import scipy.sparse as ss\n",
    "\n",
    "#相似度/距离\n",
    "import scipy.spatial.distance as ssd\n",
    "\n",
    "from collections import defaultdict\n",
    "from sklearn.preprocessing import normalize"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 统计训练集中有多少不同的用户的events\n",
    "uniqueUsers = set()\n",
    "uniqueEvents = set()\n",
    "\n",
    "#倒排表\n",
    "#统计每个用户参加的活动   / 每个活动参加的用户\n",
    "eventsForUser = defaultdict(set)\n",
    "usersForEvent = defaultdict(set)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Number of uniqueusers :3391\n",
      "Number of uniqueevents :13418\n"
     ]
    }
   ],
   "source": [
    "for filename in ['/Users/cuiyue/Desktop/AI/第四周/作业/要求/71-57-1-1519640547/train.csv',\n",
    "                 '/Users/cuiyue/Desktop/AI/第四周/作业/要求/71-57-1-1519640547/test.csv']:\n",
    "    f = open(filename,'rb')\n",
    "\n",
    "    # 忽略第一行列名字\n",
    "    f.readline().decode('utf-8').strip().split(',')\n",
    "    for line in f: # 对每一条记录\n",
    "        cols = line.decode('utf-8').strip().split(',')\n",
    "        uniqueUsers.add(cols[0]) # 第一列为用户ID\n",
    "        uniqueEvents.add(cols[1]) # 第二列为活动ID\n",
    "        \n",
    "    f.close\n",
    "    \n",
    "n_uniqueUsers = len(uniqueUsers)\n",
    "n_uniqueEvents = len(uniqueEvents)\n",
    "\n",
    "print ('Number of uniqueusers :%d' % n_uniqueUsers)\n",
    "print ('Number of uniqueevents :%d' % n_uniqueEvents)\n",
    "\n",
    "#用户关系矩阵表，可用于后续LFM/SVD++处理的输入\n",
    "#这是一个稀疏矩阵，记录用户对活动感兴趣\n",
    "\n",
    "userEventScores = ss.dok_matrix((n_uniqueUsers, n_uniqueEvents))\n",
    "userIndex = dict()\n",
    "eventIndex = dict()\n",
    "\n",
    "#重新编码用户索引字典\n",
    "for i, u in enumerate(uniqueUsers):\n",
    "    userIndex[u] = i\n",
    "\n",
    "#重新编码活动索引字典  \n",
    "for i, e in enumerate(uniqueEvents):\n",
    "    eventIndex[e] = i"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "n_records = 0\n",
    "ftrain = open('/Users/cuiyue/Desktop/AI/第四周/作业/要求/71-57-1-1519640547/train.csv','r')\n",
    "ftrain.readline()\n",
    "for line in ftrain:\n",
    "    cols = line.strip().split(',')\n",
    "    iu = userIndex[cols[0]] # 返回训练集中用户在index中的索引号\n",
    "    ie = eventIndex[cols[1]] # 返回训练集中活动在index中的索引号\n",
    "    \n",
    "    eventsForUser[iu].add(ie) #该用户参加了这个活动\n",
    "    usersForEvent[ie].add(iu) #该活动被用户参加\n",
    "    \n",
    "    score = int(cols[4])\n",
    "    userEventScores[iu, ie]=score\n",
    "ftrain.close()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<3391x13418 sparse matrix of type '<class 'numpy.float64'>'\n",
       "\twith 4131 stored elements in Dictionary Of Keys format>"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "userEventScores"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "##统计每个用户参加的活动，后续用于将用户朋友参加的活动影响到用户\n",
    "# 使用pickle类来进行python对象的序列化\n",
    "pickle.dump(eventsForUser, open('PE_eventsForUser.pkl','wb'))\n",
    "##统计活动参加的用户\n",
    "pickle.dump(usersForEvent, open(\"PE_usersForEvent.pkl\", 'wb'))\n",
    "#保存用户-活动关系矩阵R，以备后用\n",
    "sio.mmwrite(\"PE_userEventScores\", userEventScores)\n",
    "\n",
    "#保存用户索引表\n",
    "pickle.dump(userIndex, open('PE_userIndex.pkl', 'wb'))\n",
    "#保存活动索引表\n",
    "pickle.dump(eventIndex, open('PE_eventIndex.pkl', 'wb'))\n",
    "\n",
    "# 为了防止不必要的计算，我们找出来所有关联的用户 或者 关联的event\n",
    "# 所谓的关联用户，指的是至少在同一个event上有行为的用户pair\n",
    "# 关联的event指的是至少同一个user有行为的event pair\n",
    "\n",
    "uniqueUserPairs = set()\n",
    "uniqueEventPairs = set()\n",
    "for event in uniqueEvents:\n",
    "    i = eventIndex[event]\n",
    "    users = usersForEvent[i]\n",
    "    if len(users) > 2:\n",
    "        uniqueUserPairs.update(itertools.combinations(users,2))\n",
    "for user in uniqueUsers:\n",
    "    u = userIndex[user]\n",
    "    events = eventsForUser[u]\n",
    "    if len(events) > 2:\n",
    "        uniqueEventPairs.update(itertools.combinations(events,2))\n",
    "        \n",
    "#保存用户-事件关系对索引表\n",
    "pickle.dump(uniqueUserPairs, open('PE_uniqueUserPairs.pkl', 'wb'))\n",
    "pickle.dump(uniqueEventPairs, open('PE_uniqueEventPairs.pkl', 'wb'))\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
